from filecmp import cmp

import datetime as datetime
import pandas as pd
from matplotlib import pyplot as plt
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import Lasso
from sklearn.ensemble import RandomForestRegressor
from scipy.stats import pearsonr
import datetime

train = pd.read_csv("Tap4Fun/data/train_sample.csv")
train = train[train['prediction_pay_price'] != 0]
train = train[train['prediction_pay_price'].notnull()]
train = train.sort_values(by='prediction_pay_price', ascending=False)
train.to_csv('Tap4Fun/data/predictionNotNull.csv')
plt.figure(figsize=(10, 8))
train.plot.scatter(x='pay_price', y='prediction_pay_price')
plt.show()

# already drop prediction's 0 value's dataset
train = pd.read_csv('Tap4Fun/data/predictionNotNull.csv')
# feature engineer
sds = StandardScaler()
sds.fit(train)
train_sds = sds.transform(train)

lasso = Lasso()
lasso.fit(train['pay_price'], train['prediction_pay_price'])

rf = RandomForestRegressor()
rf.fit(train, train)
rf.feature_importances_

pearsonr(train, train['prediction_pay_price'])
# recursion every line
for index, row in train.iterrows():
    print(index)
    print(row)

train = train.reset_index()

##################################################################################
# manipulate the register_time attribute
train = pd.read_csv("Tap4Fun/data/predictionNotNull.csv")
server_start_date = datetime.datetime.strptime('2018-01-26 00:00:00', '%Y-%m-%d %H:%M:%S')

open_server_train = pd.DataFrame(columns=train.columns)
#first:iterrows could just check the data,couldn't change the data.
#this method is too slow by iterrows
for index, row in train.iterrows():
    # print(row.register_time)
    d = datetime.datetime.strptime(row.register_time, '%Y/%m/%d %H:%M')
    # print((d-server_start_date).seconds)
    row['time_from_open_server'] = (d - server_start_date).seconds
    print (row['time_from_open_server'])
    open_server_train = open_server_train.append(row,ignore_index=True)
#second:use method
#change str to datetime format
train['register_time'] = pd.to_datetime(train['register_time'])
def getstartday(train):
    server_start_date = datetime.datetime.strptime('2018-01-26 00:00:00', '%Y-%m-%d %H:%M:%S')
    return (train['register_time'] - server_start_date).seconds
train.set_index('user_id')
train['time_from_open_server'] = train.apply(lambda train:getstartday(train),axis=1)
train.to_csv('Tap4Fun/data/predictionNotNull.csv')
##################################################################################
# calculate p-value and correlation coefficient value byrecursion every column
columnsPearsonr = pd.DataFrame(columns=['column', 'corcoe', 'p_value'])
train_pearsonr = train.drop('register_time', axis=1)
for col_name in train_pearsonr.columns:
    # print(col_name)
    # print(pearsonr(train_pearsonr['prediction_pay_price'],train_pearsonr[col_name]))
    # get every attribute's pearsonr
    cp = pearsonr(train_pearsonr['prediction_pay_price'], train_pearsonr[col_name])
    #insert new attribute's pearsonr
    insertRow = pd.DataFrame([[col_name,cp[0],cp[1]]],columns=['column', 'corcoe', 'p_value'])
    columnsPearsonr = columnsPearsonr.append(insertRow,ignore_index=True)
columnsPearsonr.to_csv("Tap4Fun/data/corcoeAndpValue.csv")
##################################################################################
#sort the correlation coefficient
columnsPearsonr = pd.read_csv("Tap4Fun/data/corcoeAndpValue.csv")
columnsPearsonr = columnsPearsonr[columnsPearsonr['corcoe'].notnull()]
#set the max rows that pandas show
pd.set_option("max_rows",200)
columnsPearsonr = columnsPearsonr.sort_values(by='corcoe',ascending=False)

##################################################################################
#filter the middle and high correaltion coefficient attributes,then filter the dataset and save it.
train = train[columnsPearsonr[columnsPearsonr['corcoe']>0.5]['column']]
train.to_csv("Tap4Fun/data/dataWithAttributesMoreThanhalfOne.csv")
train = pd.read_csv("Tap4Fun/data/dataWithAttributesMoreThanhalfOne.csv")

#get every attribute's describe
for column in train.columns:
    print(column)
    print(train[column].describe())

##################################################################################
#handle the outliers
train = train[train['prediction_pay_price']<7000]
del train['Unnamed: 0']
train.to_csv("Tap4Fun/data/handleOutliers.csv")
train = pd.read_csv("Tap4Fun/data/handleOutliers.csv")